﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Text;
using System.Threading;
using Apache.Cassandra;
using HectorSharp;
using HectorSharp.Utils;
using HectorSharp.Utils.ObjectPool;
using System.IO;
using System.Text.RegularExpressions;
using Thrift.Protocol;
using Thrift.Transport;
using Column = HectorSharp.Column;
using ColumnOrSuperColumn = HectorSharp.ColumnOrSuperColumn;
using ColumnParent = HectorSharp.ColumnParent;
using ColumnPath = HectorSharp.ColumnPath;
using ConsistencyLevel = HectorSharp.ConsistencyLevel;
using SlicePredicate = HectorSharp.SlicePredicate;
using SliceRange = HectorSharp.SliceRange;
using SuperColumn = Apache.Cassandra.SuperColumn;

namespace LarasMutiaraDiva.Skripsi.Libs
{
    public class CassandraApp
    {
        public static int theRealCorpusNumber = 1000;

        public static int NumbersOfCorpus =1000;
        //public static int NumbersOfKey = 15496;
        public static int NumbersOfProcessedCorpus=0;
        //public static int NumbersOfProcessedKey = 0;
        public static string ProcessingFile = ""; 

        internal ICassandraClient Client;
        internal IKeyspace Keyspace;
        internal IKeyedObjectPool<Endpoint, ICassandraClient> Pool;
        public static string korpusDirectory = ConfigurationManager.AppSettings["corpusLocation"];
        public static string cassandraHost = ConfigurationManager.AppSettings["cassandraHost"];
        public static int cassandraPort = Convert.ToInt32(ConfigurationManager.AppSettings["cassandraPort"]);
        public static string cassandraKeyspace= ConfigurationManager.AppSettings["cassandrakeyspace"];


        public static Dictionary<string, double> GetRelevanDocuments(string userQuery, int numbersOfrelevanDocs)
        {
            var relevanDocs = new Dictionary<string, double>();
            var Documents = DataBaseHelper.GetAllDocuments();
            var kueri = getTermKueri(userQuery);
            foreach (var document in Documents)
            {
                relevanDocs.Add(document, GetSimilarity(kueri, document));
            }
            return relevanDocs;


        }
        public static void Main()
        {
            //Console.WriteLine(DataBaseHelper.GetDocLength("balipost23102009.txt"));
            //Console.ReadKey();
           // var all = GetAllTerms();
            bool enter = true;
            while (enter)
            {
                Menu();
                Console.Write("Enter an option: ");
                int input = Convert.ToInt32(Console.ReadLine());
                switch (input)
                {
                    case 0:
                        enter = false;
                        break;
                    case 1:
                        DataBaseHelper.TestSqlServerConnection();
                        Pause();
                        break;
                    case 2:
                        TestCassandraConnection();
                        Pause();
                        break;
                    case 3:
                        Indexing();
                        Pause();
                        break; 
                    case 4:
                        DisplayTermVariables();
                        Pause();
                        break;
                    case 5:
                        UpdateTfIdf();
                        Pause();
                        break;
                    case 6:
                        //DataBaseHelper.InsertDocLength();
                        DataBaseHelper.UpdateDocLength();
                        Pause();
                        break;
                    case 7:
                        DataBaseHelper.GetSortedDoc();
                        Pause();
                        break;
                    case 8:
                        SearchTest();
                        Pause();
                        break;
                        
                }
            }
        }

        private static void SearchTest()
        {
            Console.Write("Kueri: ");
            string userQuery = Console.ReadLine();
            var start = DateTime.Now;
            Dictionary<string, double> result =  GetRelevanDocuments(userQuery, 10);
            Console.WriteLine(DateTime.Now - start);
            var items = result.OrderByDescending(i => i.Value);
            foreach (var d in items )
            {
                Console.WriteLine("{0} => {1}", d.Key, d.Value);
                Console.ReadLine();
            }

           
        }

        public static void Menu()
        {
            Console.Clear();
            Console.WriteLine("1. Test SQL Server Connection");
            Console.WriteLine("2. Test Cassandra Connection");
            Console.WriteLine("3. Index Corpus");
            Console.WriteLine("4. Get df,idf value by term, get tf - tf.idf value by term and doc, get document length by certain doc, and get similarity by document ");
            Console.WriteLine("5. Update tf.idf values ");
            Console.WriteLine("6. Update Document Length");
            Console.WriteLine("7. Get Top Doc");
            Console.WriteLine("8. Search Test");

            Console.WriteLine("0. Exit");
        }

        public static void Pause()
        {
            Console.WriteLine("Press any key to continue...");
            Console.ReadKey();
        }

        private static void TestCassandraConnection()
        {
            CassandraApp app = new CassandraApp();
            bool connect = false;
            try
            {
                app.Pool = new CassandraClientPoolFactory().Create();
                app.Client = new KeyedCassandraClientFactory(app.Pool,
                                                             new KeyedCassandraClientFactory.Config {Timeout = 10})
                    .Make(new Endpoint(cassandraHost, cassandraPort));
                Console.WriteLine("Connection to cassandra success");
                connect = true;
            }
            catch (Exception error)
            {
                Console.WriteLine("Unable connect to cassandra server\nError: " + error.Message);
            }

            if (connect) //check if can be connected to cassandra then know that the keyspace is found or not
            {
                try
                {
                    app.Keyspace = app.Client.GetKeyspace(
                        cassandraKeyspace,
                        ConsistencyLevel.ONE,
                        new FailoverPolicy(0) {Strategy = FailoverStrategy.FAIL_FAST});
                    Console.WriteLine("Found keyspace " + cassandraKeyspace);
                }
                catch (Exception err)
                {

                    Console.WriteLine("Unable to find keyspace: " + cassandraKeyspace + "\nERROR: " + err.Message);
                }

            }
        }

        public static CassandraApp GetCassandraApp()  //connect client to cassandra and set keyspace of casssandra
        {
            CassandraApp app = new CassandraApp();
            app.Pool = new CassandraClientPoolFactory().Create();

            app.Client = new KeyedCassandraClientFactory(app.Pool,
                                new KeyedCassandraClientFactory.Config { Timeout = 100000000 })
                            .Make(new Endpoint(cassandraHost, cassandraPort));


            app.Keyspace = app.Client.GetKeyspace(
                cassandraKeyspace,
                ConsistencyLevel.ONE,
                new FailoverPolicy(0) { Strategy = FailoverStrategy.FAIL_FAST });
            return app;
        }


        static void Indexing()
        {
            var app = GetCassandraApp();

            //set up column path to use

            var files = GetFileInDirectory(korpusDirectory); //file all file name in directory
            NumbersOfCorpus = files.Count();

            Thread myThread = new Thread(new ThreadStart(IndexingProgress));    //new thread for knowing the progress of indexing corpus, run method IndexingProgress
            myThread.Start();

            int n = 0;
            foreach (var file in files)
            {

                n++;
                NumbersOfProcessedCorpus = n;       //corpus that have been indexed
                ProcessingFile = file;

                var contents = GetFileContent(file);
                var words = GetWords(contents);


                var columnFamily = "Documents";      //super column family that is used

                var stemmedWords = new List<string>();

                foreach (var word in words)
                {
                    var term = DataBaseHelper.StemmingWord(word.ToLower());
                    stemmedWords.Add(term);         //add stemming word result to list stemmedWords

                }

                foreach (var term in stemmedWords)
                {
                    int tfOfTerm = GetTermFrekuensi(term, stemmedWords); //get tf of term in certain document/file (must list of string word in doc)

                    var cpTf = new ColumnPath(columnFamily, Path.GetFileName(file), "tf");           //set path ( super column family "Documents", column family "doc_name", "tf")
                    //var cpTfIdf = new ColumnPath(columnFamily, Path.GetFileName(file), "tf.idf");    //set path ( super column family "Documents", column family "doc_name", "tf.idf")
                    //

                    app.Keyspace.Insert(term, cpTf, tfOfTerm.ToString());       //insert key=term, to what path, the value of tf
                    //app.Keyspace.Insert(term, cpTfIdf, "0.75");                 //insert key=term, to what path, the value of tf.idf, because we need tf first to get the real value of tf.idf so we set the tf.idf value with any kind of number

                }

            }

        }
        
        public static int GetTermFrekuensi(string word, List<string> words)     //get term frekuensi of word in a document
        {
            int tf = 0;
            foreach (var word1 in words)
            {
                if (word == word1)
                {
                    tf++;
                }
            }
            return tf;
        }


        public static void DisplayTermVariables()           //display df and idf of term
        {
            /*
            Console.Write("Term: ");
            string termInput = Console.ReadLine();
            Console.WriteLine("df = "+GetDocumentsFrekuensi(termInput));
            Console.WriteLine("idf = "+GetIdf(termInput));
             */
            Console.Write("Doc: ");

            string docInput = Console.ReadLine();
            //Console.WriteLine("tf = " + GetTfByTermDoc(termInput, docInput));
            //Console.WriteLine("tf.Idf = " + GetTfIdfByTermDoc(termInput, docInput));
            //Console.WriteLine("docLength = " + TriftUtility.GetDocLength(docInput));
            Console.Write("Kueri: ");
            string kueri = Console.ReadLine();
            var start = DateTime.Now;
            var item = getTermKueri(kueri.ToLower());
            Console.WriteLine("Similarity : " + GetSimilarity(item, docInput) );
            Console.WriteLine("Time: "+ (DateTime.Now-start));

        }

        public static int  GetDocumentsFrekuensi(string term)
        {

            var app = GetCassandraApp();
            var columnParent = new ColumnParent("Documents");       
            //app.Keyspace.GetCount(term, columnParent);              
            return app.Keyspace.GetCount(term, columnParent);       //return numbers of document that contain term in super column "Documents", bs menggunakan GetCount dimana bisa mendapatkan banyaknya isi dari suatu key

        }

        public static  double GetIdf(string term)
        {
            int df = GetDocumentsFrekuensi(term);
            //
            if (df == 0)
                return 0.0;

            double idf = Math.Log10((double)theRealCorpusNumber/(double)df);
            return idf;
        }


        public static IDictionary<string, IList<Column>> FetchPagedResult(IKeyspace ks,
            ColumnParent cp, SlicePredicate sp, string start, string end, int rowfetchSize)
        {
            var result = ks.GetRangeSlice(cp, sp, start, end, rowfetchSize);
            return result;
        }


        public static void UpdateTfIdf()
        {
           
            var app = GetCassandraApp();
            var allTerms = TriftUtility.GetAllTerms();
            //Console.WriteLine(allTerms.Count());
            //Console.ReadKey();
            var columnFamily = "Documents";

            NumbersOfCorpus = allTerms.Count();
            NumbersOfProcessedCorpus = 0;
            Thread myThread= new Thread(new ThreadStart(IndexingProgress));
            myThread.Start();

            
            foreach (var term in allTerms)
            {
                    NumbersOfProcessedCorpus++; 
                    var idf = GetIdf(term.Key);
            
                    ProcessingFile = term.Key;
                    foreach (string doc in term.Value)
                    {
                        var cpTfIdf = new ColumnPath(columnFamily, doc, "tf.idf");
                        //var cpTfIdf = new ColumnPath(columnFamily, Path.GetFileName(file), "tf.idf"); 
                        var tfvalue = GetTfByTermDoc(term.Key, doc);
                        var tfIdf = Convert.ToDouble(tfvalue)*idf;
                        app.Keyspace.Insert(term.Key, cpTfIdf, tfIdf.ToString());
                        
                    }
            }
        }

        public static string GetTfByTermDoc(string term, string doc)
        {
            var app = GetCassandraApp();
            var cp = new ColumnPath("Documents", doc, "tf"); //Keyspace.SuperColumnFamily[Doc_name][tf] --> EkspansiKueri.Documents['Deptan26012010.txt']['tf']
         
            var column = app.Keyspace.GetColumn(term, cp); //to get column ("tf")
           // var superColumn = app.Keyspace.GetSuperColumn(term, cp);    //to get super column (doc_name)

            return column.Value;          //return tf value of term

        }

        public static List<string> GetDocumentsByQuery(Dictionary<string, double> userQuery)
        {
            var docsResult = new List<string>();
            var app = GetCassandraApp();
            //lanjutin ya ra...

            return docsResult;

        }

        public static string GetTfIdfByTermDoc(string term, string doc)
        {
            //return "0";
            var app = GetCassandraApp();
            var cp = new ColumnPath("Documents", doc, "tf.idf"); //Keyspace.SuperColumnFamily[Doc_name][tf] --> EkspansiKueri.Documents['Deptan26012010.txt']['tf']
            try
            {
                var column = app.Keyspace.GetColumn(term, cp); //to get column ("tf")
                // var superColumn = app.Keyspace.GetSuperColumn(term, cp);    //to get super column (doc_name)
                return column.Value; 
            }catch(Exception err)
            {
                return "0";
            }

                     //return tf value of term

        }

        //public static double GetSimilarity(string[] query, string doc)
        //{
        //    var app = GetCassandraApp();
        //    var allTermDoc = TriftUtility.GetAllTerms();
        //    return;
        //}

        //public static int GetTfKueri()
        //{

        //    return

        //}

        public static Dictionary<double, Dictionary<string, int>> getTermKueri(string Kueri) //mendapatkan kueri length &
                                                                                             //term kueri & tf kueri   
        {
            var termKueri = GetWords(Kueri);

            var stemmedWords = new List<string>();
            var result0 = new Dictionary<string, int>();        //term kueri & tf kueri
            var result1 = new Dictionary<double, Dictionary<string, int>>();
            foreach (var word in termKueri)
            {
                var stemmedWord = DataBaseHelper.StemmingWord(word);
                stemmedWords.Add(stemmedWord);
            }
            foreach (var word in stemmedWords)
            {
                var tfKueri = GetTermFrekuensi(word, stemmedWords);
                if(!result0.Keys.Contains(word))
                    result0.Add(word, tfKueri);

            }

            double sumTfIdfKueri = 0;
            foreach (var kueri in result0)
            {
                //sumTfIdfKueri += kueri.Value * GetIdf(kueri.Key);
                var tf_idf = kueri.Value * GetIdf(kueri.Key);
                sumTfIdfKueri += Math.Pow(tf_idf, 2);

            }
            result1.Add(Math.Sqrt(sumTfIdfKueri), result0);
            return result1;
        }

       

        public static double GetSimilarity(Dictionary<double, Dictionary<string, int>> KueriParam, string doc)
        {
            //var app = GetCassandraApp();
            //return 1.0;
            var SumTfIdfKueriDoc = 0.0;
            var columnFamily = "Documents";
            var firstParam = KueriParam.First();
            var Kueri = firstParam.Value;
            foreach (var word in Kueri)
            {
                var tfIdfKueri = GetIdf(word.Key) * word.Value;

                var tfIdfTermInDoc = GetTfIdfByTermDoc(word.Key, doc);
                SumTfIdfKueriDoc += tfIdfKueri * Convert.ToDouble(tfIdfTermInDoc);
            }

            return SumTfIdfKueriDoc/(DataBaseHelper.GetDocLength(doc)* firstParam.Key);
        }


        public static string[] GetFileInDirectory(String directory) //mendapatkan semua file yang ada pada directory
        {
            if (!System.IO.Directory.Exists(directory)) // jika direktori tidak ada
                throw
                    new System.IO.DirectoryNotFoundException();

            String[] fileName = null;
            
            fileName = System.IO.Directory.GetFiles(directory, "*.*", System.IO.SearchOption.AllDirectories); // semua file di dalam direktori diambil, jika mau .txt saja maka *.txt
            
           
            return fileName; //semua file dikembalikan
        }


        static String GetFileContent(String File) // mendapatkan isi dari file .txt
        {
            StreamReader stream = new StreamReader(File);
            string contentFile = stream.ReadToEnd(); //contentFile berisi isi dari File

            return contentFile;
        }


        static List<string> GetWords(string isiFile) // mengambil kata per kata(token) yang ada pada string yang panjang dan diletakkan di ebuah list
        {
            var temp= Regex.Split(isiFile, @"\W+"); // menghilangkan spasi dan tanda baca lainnya dan mengembalikan kata2
            var result = from i in temp             //LINQ for checking if word is not empty string
                         where i.Length > 0
                         select i;                  //get words that aren't empty
            
            return result.ToList();
        }


        public static void IndexingProgress() //multi threading for seeing percent progress
        {
            bool belumKelar = true;
            var start = DateTime.Now;
            while (NumbersOfProcessedCorpus <= NumbersOfCorpus && belumKelar)
            {


                Console.Clear();
                Console.WriteLine(string.Format("Progress: {0}/{1} ({2}) %",
                                                NumbersOfProcessedCorpus,
                                                NumbersOfCorpus,
                                                ((float)NumbersOfProcessedCorpus / (float)NumbersOfCorpus) * 100
                                      ));

                Console.WriteLine(
                    string.Format("Current file: {0}", ProcessingFile)
                    );

                Console.WriteLine("\n\n\nTime: {0}", DateTime.Now - start );
                Thread.Sleep(10);
                //
                if (NumbersOfProcessedCorpus == NumbersOfCorpus)
                    belumKelar = false;
            }
        }

        //public static double GetDocLength(string doc)
        //{
        //    var app = GetCassandraApp();
        //    var allTermDoc = TriftUtility.GetAllTerms();

        //    double docLength = 0;
        //    foreach (var term in allTermDoc)
        //    {

        //        try
        //        {
        //            var cp = new ColumnPath("Documents", doc, "tf.idf");
        //            var column = app.Keyspace.GetColumn(term.Key, cp);

        //            docLength += Math.Pow(Convert.ToDouble(column.Value), 2);
        //        }
        //        catch (Exception)
        //        {

        //            //throw;
        //            docLength += 0;
        //        }


        //    }
        //    return Math.Sqrt(docLength);

        //}
        
    }
}
